{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "12bb0743-0ca7-439b-a84d-46f6521a82ca",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['how',\n",
       " 'owa',\n",
       " 'war',\n",
       " 'are',\n",
       " 'rey',\n",
       " 'eyo',\n",
       " 'you',\n",
       " 'oui',\n",
       " 'uia',\n",
       " 'iam',\n",
       " 'amf',\n",
       " 'mfi',\n",
       " 'fin',\n",
       " 'ine',\n",
       " 'net',\n",
       " 'eth',\n",
       " 'tha',\n",
       " 'han',\n",
       " 'ank',\n",
       " 'nks']"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import re\n",
    "from simhash import Simhash\n",
    "\n",
    "def get_features(s):\n",
    "    width = 3\n",
    "    s = s.lower()\n",
    "    s = re.sub(r'[^\\w]+', '', s)\n",
    "    return [s[i:i + width] for i in range(max(len(s) - width + 1, 1))]\n",
    "\n",
    "get_features('How are you? I am fine. Thanks.')\n",
    "#print(Simhash(get_features('How are you? I am fine. Thanks.')).value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "6af6ca4d-32ea-44ad-b4d5-4bf112d06314",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3419227768443824139"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Simhash('How are you? I am fine. Thanks.').value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "14add59b-689a-4cdf-9697-8316cc860e50",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'移舟泊烟渚日暮客愁新野旷天低树江清月近人'"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s = '移舟泊烟渚，   日暮客愁新。\\n 野旷天低树，江清月近人。'\n",
    "s = re.sub(r'[^\\w]+', '', s)\n",
    "s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "cd649554-6983-4fdd-a6eb-668067861c18",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('James', 'Bond'),\n",
       " ('Ann', 'Varsa'),\n",
       " ('Tom Cruise', 'XXX'),\n",
       " ('Tom Brand', None)]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from pyspark.sql import SparkSession\n",
    "spark = SparkSession.builder.appName('SparkByExamples.com').getOrCreate()\n",
    "\n",
    "data=[(\"James\",\"Bond\",\"100\",None),\n",
    "      (\"Ann\",\"Varsa\",\"200\",'F'),\n",
    "      (\"Tom Cruise\",\"XXX\",\"400\",''),\n",
    "      (\"Tom Brand\",None,\"400\",'M')] \n",
    "columns=[\"fname\",\"lname\",\"id\",\"gender\"]\n",
    "df=spark.createDataFrame(data,columns)\n",
    "\n",
    "#alias\n",
    "from pyspark.sql.functions import expr\n",
    "l = []\n",
    "for t in df.select(df.fname, df.lname).collect():\n",
    "    l.append(tuple(t))\n",
    "l"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "010d270f-d135-458c-b2bb-6f1c343fc7b3",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = [(\"1\",\"移舟泊烟渚，日暮客愁新\"),\n",
    "        (\"2\",\"野旷天低树，江清月近人\"),\n",
    "        (\"3\",\"两个黄鹂鸣翠柳，一行白鹭上晴天\"),\n",
    "        (\"4\",\"两个黄鹂鸣翠柳，一行白鹭上青天\"),\n",
    "        (\"5\",\"白日依山尽，黄河入海流，欲穷千里目，更上一层楼\"),\n",
    "        (\"6\",\"呃呃呃，曲项向天歌\"),\n",
    "        (\"7\",\"鹅鹅鹅，曲项向天歌\"),\n",
    "        (\"8\",\"以正合，以奇胜，故善出奇者，无穷如天地，不绝如江海\"),\n",
    "        (\"9\",\"移舟泊烟渚，日暮客愁新\")\n",
    "       ]\n",
    "columns=[\"create_id\",\"content\"]\n",
    "df=spark.createDataFrame(data,columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "6bdfadcc-6f6b-430b-9b4a-5d1898bc7c9a",
   "metadata": {},
   "outputs": [],
   "source": [
    "from simhash import Simhash,SimhashIndex"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "d0534078-b0a6-4957-9dfa-29b0502c7180",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('1', <simhash.Simhash at 0x7fdf4093e3d0>),\n",
       " ('2', <simhash.Simhash at 0x7fdf48929c70>),\n",
       " ('3', <simhash.Simhash at 0x7fdf48929790>),\n",
       " ('4', <simhash.Simhash at 0x7fdf48929940>),\n",
       " ('5', <simhash.Simhash at 0x7fdf488a33d0>),\n",
       " ('6', <simhash.Simhash at 0x7fdf488b34c0>),\n",
       " ('7', <simhash.Simhash at 0x7fdf488b3cd0>),\n",
       " ('8', <simhash.Simhash at 0x7fdf48897400>),\n",
       " ('9', <simhash.Simhash at 0x7fdf48897c10>)]"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result = df.rdd.map(lambda x: (x[0],Simhash(x[1]))).collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "id": "a5664a2e-80e4-4f5d-b1e6-0fd9666bc973",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "index = SimhashIndex(result, k = 6)\n",
    "s = '两个黄鹂鸣翠柳，一行白鹭上晴天1'\n",
    "s1 = Simhash(s)\n",
    "simi_list = index.get_near_dups(s1)\n",
    "simi_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "id": "f9fd3e21-4b7e-4277-8d47-1e3e22544e7c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---+-------------------------------------+---+\n",
      "| _1|                                   _2| _3|\n",
      "+---+-------------------------------------+---+\n",
      "|  1|               移舟泊烟渚，日暮客愁新|1,9|\n",
      "|  2|               野旷天低树，江清月近人|  2|\n",
      "|  3|       两个黄鹂鸣翠柳，一行白鹭上晴天|  3|\n",
      "|  4|       两个黄鹂鸣翠柳，一行白鹭上青天|  4|\n",
      "|  5|白日依山尽，黄河入海流，欲穷千里目...|  5|\n",
      "|  6|                   呃呃呃，曲项向天歌|  6|\n",
      "|  7|                   鹅鹅鹅，曲项向天歌|  7|\n",
      "|  8|以正合，以奇胜，故善出奇者，无穷如...|  8|\n",
      "|  9|               移舟泊烟渚，日暮客愁新|1,9|\n",
      "+---+-------------------------------------+---+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df.rdd.map(lambda x: (x[0],x[1],\",\".join(index.get_near_dups(Simhash(x[1]))))).toDF().show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "9ac0036e-4efc-4611-9372-3186ac80cbd7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "24\n"
     ]
    }
   ],
   "source": [
    "print(index.bucket_size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "id": "0faaa96b-1304-4b20-8244-cd3ce6e35d18",
   "metadata": {},
   "outputs": [],
   "source": [
    "s = '两个黄鹂鸣翠柳，一行白鹭上晴天1'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "id": "7244ca5f-e351-4def-9862-72dff180decc",
   "metadata": {},
   "outputs": [],
   "source": [
    "s1 = Simhash(s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "93513b76-0124-4db6-a215-3651a34d2cfe",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "simi_list = index.get_near_dups(s1)\n",
    "simi_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "671ffd0b-1b34-4dd7-b950-3234ba584e22",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
